{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Comparison of value and policy iteration algorithm (Reinforcement learning)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Imports"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "from value_iteration import valueIteration\n",
    "from policy_iteration import policyIteration\n",
    "import matplotlib.pyplot as plt\n",
    "import numpy as np"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Computation"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "thetas = np.array([1e-1, 1e-5, 1e-10, 1e-15, 1e-20, 1e-25, 1e-30, 1e-35, 1e-40])\n",
    "gamma = 1\n",
    "living_reward, goal_reward, death_reward = 0.05, 1, -1\n",
    "r = [living_reward, goal_reward, death_reward]\n",
    "tv, epv, tp, epp = [], [], [], []"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "for theta in thetas:\n",
    "    ep1, t1 = valueIteration(gamma, theta, r, silent = True)\n",
    "    tv.append(t1)\n",
    "    epv.append(ep1)\n",
    "    ep2, t2  = policyIteration(gamma, theta, r, silent = True)\n",
    "    tp.append(t2)\n",
    "    epp.append(ep2)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Plots"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7febc7af0da0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAXQAAAEICAYAAABPgw/pAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAIABJREFUeJzt3Xt4FdW9//H31xBMSlBEgoJEwVppMUCAEG2Vo4iI7VERBZSHtnhBwBYvLeV4fSpaWi9o1dpjxVYLVitSKkIvHrVcqrYoSTACYj1yFH/cE1EoSJQA398fM4kb2El2kr1zmXxez7OfzKyZWeu7V3a+mb3mZu6OiIi0fIc1dQAiIpIcSugiIhGhhC4iEhFK6CIiEaGELiISEUroIiIRoYQuNTKzWWY2vYnaNjP7rZl9YmbLk1z3WDN7Kcl1djczN7M2yaw3VcJYT2rqOCR5lNBbGDNbZ2alZtYupmy8mS1twrBS5QxgKNDN3QuSWbG7P+3u5yazzubMzJaa2fgk1qd/Bs2QEnrLlAZc39RB1JWZpdVxkxOAde7+aSriEYkaJfSWaQbwIzPrcPCCeF/7Y/fOzOxyM/uHmT1gZtvN7H0z+0ZYvj7c+x93ULWdzOxlM9tpZn83sxNi6v5quOxjM3vXzEbHLJtlZr8ys7+a2afA4DjxdjWzheH2a83s6rD8KuA3wNfNbJeZ3RGvI8zsSjN7JxyWefGg2NzMrgvf40dmNsPMDovph9fCaQv7o9TM/m1mq8wsN1x2pJk9aWZlZvahmd0WU0eamd0X1v0+8J8HxXakmT1uZpvNbKOZTa/8p2ZmJ4V9uSPc/tlq3t8LZjb5oLK3zOzimuI+aP2fAoOAX4Z9+cuYxeeY2XvhZ+G/zcxq61szeyVc5a2wvkvN7Cgz+3PYT5+E093ivSdJIXfXqwW9gHXAOcBzwPSwbDywNJzuDjjQJmabpcD4cPpyYC9wBcGe/nTg/wH/DRwOnAvsBLLC9WeF8/8RLn8IeC1c1g5YH9bVBugHfAT0itl2B3A6wc5DRpz38wrwCJAB5AFlwNkxsb5WQ18MB9YCXwvbvw34Z8xyB5YAHYHjgf89qB8q38cwoBjoAFhYX5dw2ZPAAqB92Lf/C1wVLpsE/AvICdtYEtv3wHxgZthPnYHlwMRw2TPArZX9ApxRzXv8LvCPmPlewPbwd1Ft3HHqqfoMHNQ/fw63Pz7s+/Pq0LcnxcwfDVwCfCnsqz8Azzf130trezV5AHrV8Rf2RULPDZNlNnVP6O/FLOsdrn9MTNk2IC+cngXMiVmWBewLk9ilwKsHxTcTuD1m2ydreC85YV3tY8ruAmbFxFpTQn+hMrmG84cBu4ETwnmvTFDh/PeARQfXDZxNkKhPAw6LWT8N2EP4DyosmxjT14uBSTHLzq3se+AY4HMgM2b5GGBJOP0k8BjB8YGaft/tgU9j3tNPgSdqiruaeqo+AzFlTsw/EmAucFMd+vakGtrLAz5p6r+X1vbSkEsL5e6rCfaubqrH5ltjpsvD+g4uy4qZXx/T7i7gY6ArwRj3qeHX9e1mth0YCxwbb9s4ugIfu/vOmLIPgeMSfB8nAA/FtP0xwZ5q7Pax7X8YtnkAd18M/JLgW0qpmT1mZkcAnYD0cLt48XWNU39sbOnA5pj4ZhLsqQP8VxjrcjN728yujPcGw775C3BZWDQGeLqWuOtiS8z0br74vSfSt1XM7EtmNjMclvo3wTevDlb34ybSAEroLdvtwNUc+EdWeQDxSzFlsQm2PnIqJ8wsi2B4YRNBMvu7u3eIeWW5+zUx29Z0O89NQEczax9TdjywMcG41hMMYcS2n+nu/4wXe1j3pngVufsv3H0AwZDGycBUguGjCoLkFi++zXHqj43tc6BTTGxHuPspYXtb3P1qd+9KsNf/iFV/1sgzwBgz+zrB8MySWuKO+xarKa9OIn0bawrQEzjV3Y8gGKKD4J+ANBIl9BbM3dcCzwLXxZSVESScb4cH7a4EvtzApr5lZmeYWVvgJ8Dr7r6e4BvCyWb2HTNLD18DzexrCca/HvgncJeZZZhZH+Aq4KkE43oUuNnMToGqg5CjDlpnanjALofgzKBDDj6GMZ9qZukE/xA/A/a7+z6CYYifmln78KDgD2PimwtcZ2bdzOwoYr4tuftm4CXgfjM7wswOM7Mvm9mZYZujYg4afkKQcPdX8z7/SvBP5U7gWXffX1Pc1dSxFTixmmXx1Na3B9fXnuCb3XYz60iwsyGNTAm95buT4KBbrKsJ9tS2AacQJM2G+D3BH+jHwADg21A1HHAuwXDAJoKv7/cQHLBL1BiCcf9NBAcRb3f3vyWyobvPD9ubE37NXw1886DVFhAcOCwhGLp4PE5VRwC/JkisHxL024xw2bUEyfJ94DWCvngiXPZr4EXgLWAFwYHqWN8F2gJrwrrnAV3CZQOBN8xsF7AQuN7d36/mfX4e1n1O2H4icR/sIWBkeAbKL6pZJ7bN2vp2GjA7HJIZDTwIZBJ8q3kd+J/a2pDkM3c94EKiycwc+Er4TUYk8rSHLiISEUroIiIRoSEXEZGI0B66iEhENOptPjt16uTdu3dvzCZFRFq84uLij9w9u7b1GjWhd+/enaKiosZsUkSkxTOzD2tfS0MuIiKRoYQuIhIRSugiIhHRIp59KCJ1U1FRwYYNG/jss8+aOhSpg4yMDLp160Z6enq9tldCF4mgDRs20L59e7p3707MQ4ikGXN3tm3bxoYNG+jRo0e96tCQi0gEffbZZxx99NFK5i2ImXH00Uc36FuVErpIRCmZtzwN/Z0poYuIRETLSugr58IDuTCtQ/Bz5dymjijQXOOC5hub4oq0wYMH8+KLLx5Q9uCDD3LNNddUs0UgKyurxuWJevTRR3nyyScBmDVrFps2xX1QVb0sXbqUf/7zi0cMxLbV1FrOQdGVc+FP10FFeTC/Y30wD9BntOKKp7nGprgib8yYMcyZM4dhw4ZVlc2ZM4d77723UdqfNGlS1fSsWbPIzc2la9dDHidbrb1799KmTfz0uHTpUrKysvjGN75xSFtNreXsoS+684s/tEoV5UF5U2qucUHzjU1xNTvPv7mR0+9eTI+b/sLpdy/m+TcTfaxrfCNHjuQvf/kLe/bsAWDdunVs2rSJQYMGsWvXLoYMGUL//v3p3bs3CxYsOGT7pUuXcv7551fNT548mVmzZgFQXFzMmWeeyYABAxg2bBibN28+ZPtp06Zx3333MW/ePIqKihg7dix5eXmUl5dXu/1ZZ53FDTfcQH5+Pg899BB/+tOfOPXUU+nXrx/nnHMOW7duZd26dTz66KM88MAD5OXl8eqrr1a1BVBSUsJpp51Gnz59GDFiBJ988klV3TfeeCMFBQWcfPLJvPrqqw3q3+q0nIS+Y0PdyhtLc42rphiaOjbF1aw8/+ZGbn5uFRu3l+PAxu3l3PzcqgYl9Y4dO1JQUMALL7wABHvno0ePxszIyMhg/vz5rFixgiVLljBlyhQSvY13RUUF1157LfPmzaO4uJgrr7ySW2+9tdr1R44cSX5+Pk8//TQlJSW0adOmxu337NlDUVERU6ZM4YwzzuD111/nzTff5LLLLuPee++le/fuTJo0iR/84AeUlJQwaNCgA9r77ne/yz333MPKlSvp3bs3d9xxR9WyvXv3snz5ch588MEDypOp5Qy5HNkt+Aocr7wpNde4KmNojrEprmZlxovvUl6x74Cy8op9zHjxXS7qd1y9660cdhk+fDhz5szh8ceDx7m6O7fccguvvPIKhx12GBs3bmTr1q0ce+yxtdb57rvvsnr1aoYOHQrAvn376NKlSy1bJb79pZdeWjW9YcMGLr30UjZv3syePXtqPTd8x44dbN++nTPPPBOAcePGMWrUF8/VvvjiiwEYMGAA69atSzjmumg5e+hDfgzpmQeWpWcG5U2pucYFzTc2xdWsbNpeXqfyRA0fPpxFixaxYsUKdu/ezYABAwB4+umnKSsro7i4mJKSEo455phDzr1u06YN+/fvr5qvXO7unHLKKZSUlFBSUsKqVat46aWXEo6ptu3btfvieevXXnstkydPZtWqVcycObPBV90efnjw7PS0tDT27t3boLqq03ISep/RFPa+gy1ks9+NLWRT2PuOpj9Y1Vzjas6xKa5mpWuHzDqVJyorK4vBgwdz5ZVXMmbMmKryHTt20LlzZ9LT01myZAkffnjonWFPOOEE1qxZw+eff8727dtZtGgRAD179qSsrIxly5YBwRDM22+/XWMc7du3Z+fOnXXefseOHRx3XPANZfbs2XHri3XkkUdy1FFHVY2P/+53v6vaW28sLWbI5fk3N3Jz4QmUVzxUVZZZmMZdORsb9LUwqnFB841NcTUvU4f15ObnVh0w7JKZnsbUYT0bXPeYMWMYMWIEc+bMqSobO3YsF1xwAb179yY/P5+vfvWrh2yXk5PD6NGjyc3NpUePHvTr1w+Atm3bMm/ePK677jp27NjB3r17ueGGGzjllFOqjeHyyy9n0qRJZGZmsmzZsoS3nzZtGqNGjeKoo47i7LPP5oMPPgDgggsuYOTIkSxYsICHH374gG1mz57NpEmT2L17NyeeeCK//e1v69Vv9dWozxTNz8/3+j7g4vS7F7MxzlfA4zpk8o+bzm5oaPXWXOOC5hub4kq9d955h6997WsJr//8mxuZ8eK7bNpeTtcOmUwd1jPS/8Sas3i/OzMrdvf82rZtMXvoqRrna6jmGldNMTR1bIqr+bmo33FK4BHQYsbQUzXO11DNNa6aYmjq2BSXSGoknNDNLM3M3jSzP4fzPczsDTNba2bPmlnb1IUZjPNlpqcdUJascb6GaK5xQfONTXGJpEZd9tCvB96Jmb8HeMDdTwI+Aa5KZmAHu6jfcdx1cW+O65CJEYxr3nVx7yb/mthc42rOsSkukdRI6KComXUDZgM/BX4IXACUAce6+14z+zowzd2H1VBNgw6Kikji6npQVJqPhhwUTXQP/UHgv4DKM/2PBra7e+XZ8RuAuLsxZjbBzIrMrKisrCzB5kREpK5qTehmdj5Q6u7F9WnA3R9z93x3z8/Ozq5PFSLSAqWlpZGXl0dubi6jRo1i9+7dNa5feevcTZs2MXLkyAa3P378eNasWQPAz372swbXF+vgW/LGttWUEtlDPx240MzWAXOAs4GHgA5mVnnaYzegYbdnE5FIyczMpKSkhNWrV9O2bVseffTRhLbr2rUr8+bNa3D7v/nNb+jVqxdQv4S+b9++apcdnNBj22pKtSZ0d7/Z3bu5e3fgMmCxu48FlgCV/0bHAYfeA1NEkqJw4Uy2TDuJ/bcfyZZpJ1G4cGZyG0jxgz0GDRrE2rVrAfj5z39Obm4uubm5PPjgg4esu27dOnJzc4Egqf7oRz8iNzeXPn368PDDD7N48WIuuuiiqvVffvllRowYcUg9Z511FkVFRdx0002Ul5eTl5fH2LFjAXjqqacoKCggLy+PiRMnViXvrKwspkyZQt++fVm2bBl33nknAwcOJDc3lwkTJuDucW/JW9kWwDPPPEPv3r3Jzc3lxhtvrIonKyuLW2+9lb59+3LaaaexdevWJPVuDHdP+AWcBfw5nD4RWA6sBf4AHF7b9gMGDHARqZvlCx713T/u5H77EVWv3T/u5MsXPFrtNmvWrEm8gbeedZ9+zAH1+/RjgvIGaNeunbu7V1RU+IUXXuiPPPKIFxUVeW5uru/atct37tzpvXr18hUrVhyw/gcffOCnnHKKu7s/8sgjfskll3hFRYW7u2/bts3379/vPXv29NLSUnd3HzNmjC9cuPCQ9s8880wvLCw8oG73oG/OP/9837Nnj7u7X3PNNT579mx3dwf82We/eN/btm2rmv72t79d1U5s3bHzGzdu9JycHC8tLfWKigofPHiwz58/v6ruyu2nTp3qP/nJT+L2W7zfHVDkCeToOl1Y5O5L3f38cPp9dy9w95PcfZS7f560/zIiUiVnxQwybc8BZZm2h5wVM5LTQIoe7FG5V5yfn8/xxx/PVVddxWuvvcaIESNo164dWVlZXHzxxTU+7OFvf/sbEydOrHp6UMeOHTEzvvOd7/DUU0+xfft2li1bxje/+c2E41q0aBHFxcUMHDiQvLw8Fi1axPvvvw8E4/6XXHJJ1bpLlizh1FNPpXfv3ixevLjWG4EVFhZy1llnkZ2dTZs2bRg7diyvvPIKENyHpvKhHam6hW6LufRfpLXq7GUQ52Hwnf2j5DSQogd7VI6hp8IVV1zBBRdcQEZGBqNGjar2cXHxuDvjxo3jrrvuOmRZRkYGaWnBxWWfffYZ3/ve9ygqKiInJ4dp06Y16Ba66enpmAW/yFTdQrfFXPov0lqVWvyzw0qtU3IaqO4BHil4sMegQYN4/vnn2b17N59++inz588/5Kk/sYYOHcrMmTOrkt/HH38MBAdOu3btyvTp07niiitqbTc9PZ2KigoAhgwZwrx58ygtLa2qM94tfCuTd6dOndi1a9cBB2qru4VuQUEBf//73/noo4/Yt28fzzzzTKPeQlcJXaSZW99/KuV+4J01yr0t6/tPTU4Djfhgj/79+3P55ZdTUFDAqaeeyvjx46tujRvP+PHjOf744+nTpw99+/bl97//fdWysWPHkpOTk9AFVBMmTKBPnz6MHTuWXr16MX36dM4991z69OnD0KFD4z6XtEOHDlx99dXk5uYybNgwBg4cWLWs8pa8lQdFK3Xp0oW7776bwYMH07dvXwYMGMDw4cMT7Z4GazG3zxVpzQoXziRnxQw6+0eUWifW95/KwAsnVrt+na8UXTk3GDPfsSHYMx/y42b/YI/JkyfTr18/rroqpXcdaXSt4va5Iq3ZwAsnQpjAjw1fSdVndLNP4LEGDBhAu3btuP/++5s6lGZFCV1EWpzi4npduB55GkMXiajGHE6V5Gjo70wJXSSCMjIy2LZtm5J6C+LubNu2jYyMjHrXoSEXkQjq1q0bGzZsQHc4bVkyMjLo1q3+p4sqoYtEUHp6Oj169GjqMKSRachFRCQilNBFRCJCCV1EJCKU0EVEIkIJXUQkIpTQRUQiQgldRCQilNBFRCKi1oRuZhlmttzM3jKzt83sjrB8lpl9YGYl4Ssv9eGKiEh1ErlS9HPgbHffZWbpwGtm9kK4bKq7z6thWxERaSS1JvTwidO7wtn08KU7/oiINDMJjaGbWZqZlQClwMvu/ka46KdmttLMHjCzw6vZdoKZFZlZkW4UJCKSOgkldHff5+55QDegwMxygZuBrwIDgY7AjdVs+5i757t7fnZ2/IfdiohIw9XpLBd33w4sAc5z980e+Bz4LVCQigBFRCQxiZzlkm1mHcLpTGAo8C8z6xKWGXARsDqVgYqISM0SOculCzDbzNII/gHMdfc/m9liM8sGDCgBJqUwThERqUUiZ7msBPrFKT87JRGJiEi96EpREZGIUEIXEYkIJXQRkYhQQhcRiQgldBGRiFBCFxGJCCV0EZGIUEIXEYkIJXQRkYhQQhcRiQgldBGRiFBCFxGJCCV0EZGIUEIXEYkIJXQRkYhQQhcRiQgldBGRiFBCFxGJCCV0EZGIqDWhm1mGmS03s7fM7G0zuyMs72Fmb5jZWjN71szapj5cERGpTiJ76J8DZ7t7XyAPOM/MTgPuAR5w95OAT4CrUhemiIjUptaE7oFd4Wx6+HLgbGBeWD4buCglEYqISEISGkM3szQzKwFKgZeB/wO2u/vecJUNwHHVbDvBzIrMrKisrCwZMYuISBwJJXR33+fueUA3oAD4aqINuPtj7p7v7vnZ2dn1DFNERGpTp7Nc3H07sAT4OtDBzNqEi7oBG5Mcm4i0BCvnwgO5MK1D8HPl3KaOqNVK5CyXbDPrEE5nAkOBdwgS+8hwtXHAglQFKSLN1Mq58KfrYMd6wIOff7pOSb2JJLKH3gVYYmYrgULgZXf/M3Aj8EMzWwscDTyeujBFpFladCdUlB9YVlEelEuja1PbCu6+EugXp/x9gvF0EWmtdmyoW7mklK4UFZH6O7Jb3colpZTQRaT+hvwY0jMPLEvPDMql0Smhi0j99RlNYe872EI2+93YQjaFve+APqObOrJWqdYxdBGR6jz/5kZuLjyB8oqHqsoyC9O4K2cjF/WLe62hpJD20EWk3ma8+C7lFfsOKCuv2MeMF99toohaNyV0Eam3TdvL61QuqaWELiL11rVDZp3KJbWU0EWk3qYO60lmetoBZZnpaUwd1rOJImrddFBUROqt8sDnjBffZdP2crp2yGTqsJ46INpElNBFpEEu6necEngzoSEXEZGIUEIXEYkIJXQRkYhQQhcRiQgldBGRiFBCFxGJCCV0EZGIUEIXEYmIRB4SnWNmS8xsjZm9bWbXh+XTzGyjmZWEr2+lPlwREalOIleK7gWmuPsKM2sPFJvZy+GyB9z9vtSFJyIiiUrkIdGbgc3h9E4zewfQdb4iIs1MncbQzaw70A94IyyabGYrzewJMzuqmm0mmFmRmRWVlZU1KFgREalewgndzLKAPwI3uPu/gV8BXwbyCPbg74+3nbs/5u757p6fnZ2dhJBFRCSehBK6maUTJPOn3f05AHff6u773H0/8GugIHVhiohIbRI5y8WAx4F33P3nMeVdYlYbAaxOfngiIpKoRM5yOR34DrDKzErCsluAMWaWBziwDpiYkghFRCQhiZzl8hpgcRb9NfnhiIhIfelKURGRiFBCFxGJCCV0EZGIUEIXEYkIJXQRkYhQQhcRiQgldBGRiFBCFxGJCCV0EZGIUEIXEYkIJXQRkYhQQhcRiQgldBGRiFBCFxGJCCV0EZGIUEIXEYkIJXQRkYhQQhcRiQgldBGRiKg1oZtZjpktMbM1Zva2mV0flnc0s5fN7L3w51GpD1dERKqTyB76XmCKu/cCTgO+b2a9gJuARe7+FWBROC8iIk2k1oTu7pvdfUU4vRN4BzgOGA7MDlebDVyUqiBFRKR2dRpDN7PuQD/gDeAYd98cLtoCHFPNNhPMrMjMisrKyhoQqoiI1CThhG5mWcAfgRvc/d+xy9zdAY+3nbs/5u757p6fnZ3doGBFRKR6CSV0M0snSOZPu/tzYfFWM+sSLu8ClKYmRBERSUQiZ7kY8Djwjrv/PGbRQmBcOD0OWJD88EREJFFtEljndOA7wCozKwnLbgHuBuaa2VXAh8Do1IQoIiKJqDWhu/trgFWzeEhywxERkfrSlaIiIhGhhC4iEhGJjKGLiLQ4hQtnkrNiBp29jFLLZn3/qQy8cGJTh5VSSugiEjmFC2eSW3wbmbYHDI6ljCOLb6MQIp3UNeQiIpGTs2JGkMxjZNoeclbMaKKIGocSuohETmePf5uRzv5RI0fSuJTQRSRySi3+bUZKrVMjR9K4lNBFJHLW959Kubc9oKzc27K+/9QmiqhxKKGLSOQMvHAiqwdMZwvZ7HdjC9msHjA90gdEASy4UWLjyM/P96KiokZrT0QkCsys2N3za1tPe+giIhGhhC4iEhFK6CIiEaGELiISEUroIiIRoYQuIhIRSugiIhGhhC4iEhGJPCT6CTMrNbPVMWXTzGyjmZWEr2+lNkwREalNInvos4Dz4pQ/4O554euvyQ1LRETqqtaE7u6vAB83QiwiItIADRlDn2xmK8MhmaOqW8nMJphZkZkVlZXFv0exiIg0XH0T+q+ALwN5wGbg/upWdPfH3D3f3fOzs+Pfo1hERBquXgnd3be6+z533w/8GihIblgiIlJX9UroZtYlZnYEsLq6dUVEpHG0qW0FM3sGOAvoZGYbgNuBs8wsD3BgHRDtu8aLiLQAtSZ0dx8Tp/jxFMQiIiINoCtFRUQiQgldRCQilNBFRCJCCV1EJCKU0EVEIkIJXUQkIpTQRUQiQgldRCQilNBFRCJCCV1EJCKU0EVEIkIJXUQkIpTQRUQiQgldRCQilNBFRCJCCV1EJCKU0EVEIkIJXUQkImpN6Gb2hJmVmtnqmLKOZvaymb0X/jwqtWGKiEhtEtlDnwWcd1DZTcAid/8KsCicFxGRJlRrQnf3V4CPDyoeDswOp2cDFyU5LhERqaP6jqEf4+6bw+ktwDFJikdEROqpwQdF3d0Br265mU0wsyIzKyorK2tocyIiUo36JvStZtYFIPxZWt2K7v6Yu+e7e352dnY9mxMRkdrUN6EvBMaF0+OABckJR0RE6iuR0xafAZYBPc1sg5ldBdwNDDWz94BzwnkREWlCbWpbwd3HVLNoSJJjERGRBtCVoiIiEaGELiISEUroIiIRoYQuIhIRSugiIhGhhC4iEhFK6CIiEaGELiISEbVeWCQiInVTuHAmOStm0NnLKLVs1vefysALJ6a8XSV0EZEkKlw4k9zi28i0PWBwLGUcWXwbhZDypK4hFxGRJMpZMSNI5jEybQ85K2akvG0ldBGRJOrs8Z/70Nk/SnnbSugiIklUavGf+1BqnVLethK6iEgSre8/lXJve0BZubdlff+pKW9bCV1EJIkGXjiR1QOms4Vs9ruxhWxWD5jeKGe5WPBI0MaRn5/vRUVFjdaeiEgUmFmxu+fXtp720EVEIkIJXUQkIpTQRUQiokFXiprZOmAnsA/Ym8gYj4iIpEYyLv0f7N4IZ8yLiEiNNOQiIhIRDU3oDrxkZsVmNiHeCmY2wcyKzKyorCz+JbEiItJwDU3oZ7h7f+CbwPfN7D8OXsHdH3P3fHfPz86Of0msiIg0XNIuLDKzacAud7+vhnXKgA+T0mDz1gnQcYWA+iKgfviC+iJQl344wd1r3SOu90FRM2sHHObuO8Ppc4E7a9omkYCiwMyKdMZPQH0RUD98QX0RSEU/NOQsl2OA+WZWWc/v3f1/khKViIjUWb0Turu/D/RNYiwiItIAOm0xNR5r6gCaEfVFQP3wBfVFIOn90Kh3WxQRkdTRHrqISEQooYuIRIQSehKZWZ6ZvW5mJeHVsQVhuZnZL8xsrZmtNLP+TR1rqpnZtWb2LzN728zujSm/OeyHd81sWFPG2JjMbIqZuVnwYMnW9pkwsxnh52Glmc03sw4xy1rdZ8LMzgvf71ozuylpFbu7Xkl6AS8B3wynvwUsjZl+ATDgNOCNpo4jbzG4AAACkElEQVQ1xf0wGPgbcHg43zn82Qt4Czgc6AH8H5DW1PE2Qn/kAC8SXFTXqZV+Js4F2oTT9wD3tNbPBJAWvs8Tgbbh+++VjLq1h55cDhwRTh8JbAqnhwNPeuB1oIOZdWmKABvJNcDd7v45gLuXhuXDgTnu/rm7fwCsBQqaKMbG9ADwXwSfj0qt6jPh7i+5+95w9nWgWzjdGj8TBcBad3/f3fcAcwj6ocGU0JPrBmCGma0H7gNuDsuPA9bHrLchLIuqk4FBZvaGmf3dzAaG5a2tHzCz4cBGd3/roEWtri9iXEnw7QRaZz+k7D0n437orYqZ/Q04Ns6iW4EhwA/c/Y9mNhp4HDinMeNrLLX0QxugI8FQwkBgrpmd2IjhNapa+uIWguGGyKupH9x9QbjOrcBe4OnGjK21UEKvI3evNkGb2ZPA9eHsH4DfhNMbCcZRK3ULy1qsWvrhGuA5DwYMl5vZfoIbEUWuH6D6vjCz3gTjwm+Ft8joBqwID5ZHri9q+kwAmNnlwPnAkPCzARHshwSk7D1ryCW5NgFnhtNnA++F0wuB74ZnNpwG7HD3zU0RYCN5nuDAKGZ2MsGBn48I+uEyMzvczHoAXwGWN1mUKebuq9y9s7t3d/fuBF+t+7v7FlrZZ8LMziM4jnChu++OWdSqPhOhQuArZtbDzNoClxH0Q4NpDz25rgYeMrM2wGdA5UM//kpwVsNaYDdwRdOE12ieAJ4ws9XAHmBcuEf2tpnNBdYQfO3+vrvva8I4m1Jr+0z8kuBMlpfDbyuvu/skd291nwl332tmkwnOfEoDnnD3t5NRty79FxGJCA25iIhEhBK6iEhEKKGLiESEErqISEQooYuIRIQSuohIRCihi4hExP8H7AjCoD/3iFYAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(np.log(thetas), epv)\n",
    "plt.scatter(np.log(thetas), epp)\n",
    "plt.title('Number of episodes vs theta')\n",
    "plt.xlabel = \"Log of theta\"\n",
    "plt.ylabel = \"# Episodes\"\n",
    "plt.legend([\"Value iteration\", \"Policy iteration\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<matplotlib.legend.Legend at 0x7febc7a1fc18>"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAAX0AAAEICAYAAACzliQjAAAABHNCSVQICAgIfAhkiAAAAAlwSFlzAAALEgAACxIB0t1+/AAAADl0RVh0U29mdHdhcmUAbWF0cGxvdGxpYiB2ZXJzaW9uIDMuMC4wLCBodHRwOi8vbWF0cGxvdGxpYi5vcmcvqOYd8AAAH3xJREFUeJzt3X90VOW97/H3t0kgFCxUwCo/NPRqaTGEYELwtOUCh2r0LDWogHJpRcUi9uC9tZaK9dZDra2/2qJtdVFvdYE/auRwhNJlu2grovYUlQkgCl7uSZXWBAsBDZUSJMTv/WPvhCEJZALJzGb257VWVmae/cye7+yZ9ZmZZ+95trk7IiISDx/LdAEiIpI+Cn0RkRhR6IuIxIhCX0QkRhT6IiIxotAXEYkRhb5kHTNbZGbfyXQdx8rMCszMzSw307VI9lHoS7czs21m1mBme83sb2a22Mz6dNG6rzazPya3ufscd/9eV6w/HcLt86UuWpfeMOSoFPqSLhe7ex+gGBgN3JrhekRiSaEvaeXufwNWEYQ/AGa2xsyuS7p+2Kf38JPrHDP7LzOrN7MHLfA5YBHwT+G3iPqw/2IzuzO8PMHMaszsW2a208zeNbPJZvYvZvb/zOw9M/t20n19zMzmm9mfzWy3mS01s5Pbeyxm9qaZXZR0PdfM6szsHDPLN7MnwnXUm9k6M/tUO+t4HDgd+HX4GL6VtHiGmf3VzHaZ2W0p1vhi+L8+XN8/mdl/M7PVYd9dZvakmfXr8MmSrKTQl7QysyHAhUB1J296ETAGKAKmAeXu/iYwB1jr7n3c/UhBdiqQDwwGbgf+D/BloAQYB3zHzIaFfW8EJgPjgUHA+8CDR1jvU8D0pOvlwC53Xw/MBPoCQ4H+YZ0NrVfg7l8B/kr4Tcjd701a/EVgODAJuD18k+uoxv8e/u8Xrm8tYMBdYd/PhTUtOMJjkiyn0Jd0WWFmHwDvADuBf+vk7e9293p3/yvwPEnfFFLQCHzf3RuBSmAA8IC7f+Dum4EtwKiw7xzgNnevcfcPCcJxyhHGyH8JXGJmHw+v/w+CN4Lm++wPnOnuTe5e5e5/70TNAN919wZ3fw147RhrxN2r3f337v6hu9cBPyZ4w5AYUuhLukx295OACcBnCYK3M/6WdHkf0JkdwbvdvSm83Pxpe0fS8oak9Z0BLA+HZOqBN4EmoM3QjLtXh8svDoP/EoI3AoDHCYaxKs1su5nda2Z5nagZjvyYU64RwMw+ZWaVZlZrZn8HnqDz21+yhEJf0srdXwAWAz9Mav4H8PGk66d2ZpVdUFayd4AL3b1f0l++u9ceoX/zEE8FsCV8I8DdG939u+4+Avg8wfDUVV30GI5WY3vr+kHYPtLdP0EwtGWdvE/JEgp9yYT7gfPMrHm4YiNwmZl93MzOBGZ1Yl07gCFm1qOLalsEfN/MzgAws4FmVnGU/pXA+cANHPqUj5lNNLORZpYD/J1guOejI6xjB/DpLqqxLryf5PWdBOwF9pjZYGBeJ+5LsoxCX9IuHFd+jGCnKsBC4ABB+C0BnuzE6lYDm4G/mdmuLijvAWAl8LtwH8TLwNgjdXb3d4G1BJ/mn05adCqwjCDw3wReIBjyac9dwP8Oh2u+eTw1uvs+4PvAf4brOxf4LnAOsAd4FngmhfuQLGU6iYqISHzok76ISIwo9EVEYkShLyISIwp9EZEYidxMfAMGDPCCgoJMlyEickKpqqra5e4DO+oXudAvKCggkUhkugwRkROKmf0llX4a3hERiRGFvohIjCj0RURiJHJj+iKSPo2NjdTU1LB///5MlyIpys/PZ8iQIeTldXbS1kBKoW9mFxDM95ED/MLd7261vCfBXColwG7gCnfflrT8dII5yxe4e/LsiiKSQTU1NZx00kkUFBRgpok3o87d2b17NzU1NQwbNqzjG7Sjw+GdcJbABwnOdjQCmG5mI1p1mwW87+5nEkyedU+r5T8GfntMFYpIt9m/fz/9+/dX4J8gzIz+/fsf1zezVMb0y4Bqd3/L3Q8QTCXbeqrZCoLZESGYWXCSha8iM5sMvE0wE6KIRIwC/8RyvM9XKqE/mOCkDc1qwrZ2+7j7QYIpXPubWR/gFoKpXY/IzGabWcLMEnV1danWLiIindTdR+8sABa6+96jdXL3h9291N1LBw7s8AdlIpIlJk6cyKpVqw5ru//++7nhhhuOers+fTpztswjW7RoEY899hgAixcvZvv27V2yXoA1a9bwpz/9qd37yqRUduTWAkOTrg8J29rrUxOenLkvwQ7dsQQnbL4X6Ad8ZGb73f1nx125iJzwpk+fTmVlJeXl5S1tlZWV3HvvvWm5/zlz5rRcXrx4MYWFhQwaNCjl2x88eJDc3PZjdM2aNfTp04fPf/7zbe4rk1L5pL8OOMvMhoWnpLuS4Kw9yVYCM8PLU4DVHhjn7gXuXkBwirwfKPBFTlwrNtTyhbtXM2z+s3zh7tWs2HCkUwenZsqUKTz77LMcOHAAgG3btrF9+3bGjRvH3r17mTRpEueccw4jR47kV7/6VZvbr1mzhosuuqjl+ty5c1m8eDEAVVVVjB8/npKSEsrLy3n33Xfb3H7BggX88Ic/ZNmyZSQSCWbMmEFxcTENDQ1HvP2ECRP4+te/TmlpKQ888AC//vWvGTt2LKNHj+ZLX/oSO3bsYNu2bSxatIiFCxdSXFzMSy+91HJfABs3buTcc8+lqKiISy+9lPfff79l3bfccgtlZWV85jOf4aWXXjqu7dueDkM/HKOfC6wiOO3bUnffbGZ3mNklYbdHCMbwq4FvAPO7vFIRyagVG2q59ZnXqa1vwIHa+gZufeb14wr+k08+mbKyMn772+DgvsrKSqZNm4aZkZ+fz/Lly1m/fj3PP/88N998M6me6a+xsZEbb7yRZcuWUVVVxbXXXsttt912xP5TpkyhtLSUJ598ko0bN5Kbm3vU2x84cIBEIsHNN9/MF7/4RV5++WU2bNjAlVdeyb333ktBQQFz5szhpptuYuPGjYwbN+6w+7vqqqu455572LRpEyNHjuS73z202/PgwYO8+uqr3H///Ye1d5WUjtN3998Av2nVdnvS5f3A1A7WseAY6hORiLhv1VYaGpsOa2tobOK+VVuZPLr1sR2pax7iqaiooLKykkceeQQIjkn/9re/zYsvvsjHPvYxamtr2bFjB6eeemqH69y6dStvvPEG5513HgBNTU2cdtppKdfU0e2vuOKKlss1NTVcccUVvPvuuxw4cKDD4+f37NlDfX0948ePB2DmzJlMnXooPi+77DIASkpK2LZtW8o1p0q/yBWRlGyvb+hUe6oqKiq46aabWL9+Pfv27aOkpASAJ598krq6OqqqqsjLy6OgoKDN8em5ubl89NFHLdebl7s7Z599NmvXrj2mmjq6fe/evVsu33jjjXzjG9/gkksuYc2aNSxYsOCY7rNZz549AcjJyeHgwYPHta72aO4dEUnJoH69OtWeqj59+jBx4kSuvfZapk+f3tK+Z88eTjnlFPLy8nj++ef5y1/azhx8xhlnsGXLFj788EPq6+t57rnnABg+fDh1dXUtod3Y2MjmzUf/qdBJJ53EBx980Onb79mzh8GDg286S5YsaWlPXl+yvn378slPfrJlvP7xxx9v+dSfDgp9EUnJvPLh9MrLOaytV14O88qHH/e6p0+fzmuvvXZY6M+YMYNEIsHIkSN57LHH+OxnP9vmdkOHDmXatGkUFhYybdo0Ro8eDUCPHj1YtmwZt9xyC6NGjaK4uPiwwyfbc/XVVzNnzhyKi4tpampK+fYLFixg6tSplJSUMGDAgJb2iy++mOXLl7fsyE22ZMkS5s2bR1FRERs3buT2229vvdpuY6nuGEmX0tJS10lURNLjzTff5HOf+1zK/VdsqOW+VVvZXt/AoH69mFc+/LjG8+XYtPe8mVmVu5d2dFuN6YtIyiaPHqyQP8FpeEdEJEYU+iIiMaLQFxGJEYW+iEiMKPRFRGJEoS8iGZWTk0NxcTGFhYVMnTqVffv2HbV/87TK27dvZ8qUKcd9/9dddx1btmwB4Ac/+MFxry9Z6+mak+8rUxT6IpJRvXr1YuPGjbzxxhv06NGDRYsWpXS7QYMGsWzZsuO+/1/84heMGBGcAfZYQr+pqemIy1qHfvJ9ZYpCX0RSt2kpLCyEBf2C/5uWdunqx40bR3V1NQA//vGPKSwspLCwkPvvv79N323btlFYWAgEwfvNb36TwsJCioqK+OlPf8rq1auZPHlyS//f//73XHrppW3WM2HCBBKJBPPnz6ehoYHi4mJmzJgBwBNPPEFZWRnFxcVcf/31LQHfp08fbr75ZkaNGsXatWu54447GDNmDIWFhcyePRt3b3e65ub7AnjqqacYOXIkhYWF3HLLLS319OnTh9tuu41Ro0Zx7rnnsmPHji7auiF3j9RfSUmJi0h6bNmyJfXOrz3tfuen3P/tE4f+7vxU0H4cevfu7e7ujY2Nfskll/hDDz3kiUTCCwsLfe/evf7BBx/4iBEjfP369Yf1f/vtt/3ss892d/eHHnrIL7/8cm9sbHR39927d/tHH33kw4cP9507d7q7+/Tp033lypVt7n/8+PG+bt26w9btHmybiy66yA8cOODu7jfccIMvWbLE3d0Bf/rpQ4979+7dLZe//OUvt9xP8rqTr9fW1vrQoUN9586d3tjY6BMnTvTly5e3rLv59vPmzfPvfe97bWpu73kDEp5CxuqTvoik5rk7oLHVjJqNDUH7cWj+dF1aWsrpp5/OrFmz+OMf/8ill15K79696dOnD5dddtlRTyjyhz/8geuvv77lLFYnn3wyZsZXvvIVnnjiCerr61m7di0XXnhhynU999xzVFVVMWbMGIqLi3nuued46623gGA/xOWXX97S9/nnn2fs2LGMHDmS1atXdzi527p165gwYQIDBw4kNzeXGTNm8OKLLwLBvEHNJ4bpjumVNQ2DiKRmT03n2lPUPKbfHa655houvvhi8vPzmTp16hFPbdged2fmzJncddddbZbl5+eTkxNMPrd//36+9rWvkUgkGDp0KAsWLGgzBXRn5OXlYWZA90yvrE/6IpKavkM6134cxo0bx4oVK9i3bx//+Mc/WL58eZuzTyU777zz+PnPf94SkO+99x4Q7OwdNGgQd955J9dcc02H95uXl0djYyMAkyZNYtmyZezcubNlne1N79wc8AMGDGDv3r2H7Vw+0vTKZWVlvPDCC+zatYumpiaeeuqptE2vrNAXkdRMuh3yWs2dn9craO9i55xzDldffTVlZWWMHTuW6667rmXa5PZcd911nH766RQVFTFq1Ch++ctftiybMWMGQ4cOTWk20dmzZ1NUVMSMGTMYMWIEd955J+effz5FRUWcd9557Z5nt1+/fnz1q1+lsLCQ8vJyxowZ07IsebrmhoZDQ2OnnXYad999NxMnTmTUqFGUlJRQUVGR6uY5LppaWSTGOju1MpuWBmP4e2qCT/iTboeiad1XYBeYO3cuo0ePZtasWZkupctoamURSY+iaZEP+WQlJSX07t2bH/3oR5kuJTIU+iKStaqqqjJdQuRoTF8k5qI2xCtHd7zPl0JfJMby8/PZvXu3gv8E4e7s3r2b/Pz8Y16HhndEYmzIkCHU1NRQV1eX6VIkRfn5+QwZcuyHySr0RWIsLy+PYcOGZboMSSMN74iIxIhCX0QkRhT6IiIxotAXEYkRhb6ISIwo9EVEYkShLyISIwp9EZEYUeiLiMSIQl9EJEYU+iIiMaLQFxGJEYW+iEiMpBT6ZnaBmW01s2ozm9/O8p5m9nS4/BUzKwjby8xsY/j3mpld2rXli4hIZ3QY+maWAzwIXAiMAKab2YhW3WYB77v7mcBC4J6w/Q2g1N2LgQuAn5uZpnMWEcmQVD7plwHV7v6Wux8AKoGKVn0qgCXh5WXAJDMzd9/n7gfD9nxAp+cREcmgVEJ/MPBO0vWasK3dPmHI7wH6A5jZWDPbDLwOzEl6ExARkTTr9h257v6Ku58NjAFuNbM2J3c0s9lmljCzhE7bJiLSfVIJ/VpgaNL1IWFbu33CMfu+wO7kDu7+JrAXKGx9B+7+sLuXunvpwIEDU69eREQ6JZXQXwecZWbDzKwHcCWwslWflcDM8PIUYLW7e3ibXAAzOwP4LLCtSyoXEZFO6/BIGnc/aGZzgVVADvCou282szuAhLuvBB4BHjezauA9gjcGgC8C882sEfgI+Jq77+qOByIiIh0z92gdUFNaWuqJRCLTZYiInFDMrMrdSzvqp1/kiojEiEJfRCRGFPoiIjGi0BcRiRGFvohIjCj0RURiRKEvIhIjCn0RkRhR6IuIxIhCX0QkRhT6IiIxolMXikhsrdhQy32rtrK9voFB/Xoxr3w4k0e3PkdUdlHoi0gsrdhQy63PvE5DYxMAtfUN3PrM6wBZHfwa3hGRWLpv1daWwG/W0NjEfau2Zqii9FDoi0gsba9v6FR7tlDoi0gsDerXq1Pt2UKhLyKxNK98OL3ycg5r65WXw7zy4RmqKD20I1dEYql5Z62O3hERiYnJowdnfci3puEdEZEYUeiLiMSIQl9EJEYU+iIiMaLQFxGJEYW+iEiMKPRFRGJEoS8iEiMKfRGRGFHoi4jEiEJfRCRGFPoiIjGi0BcRiRGFvohIjCj0RURiRKEvIhIjCn0RkRhR6IuIxIhCX0QkRhT6IiIxklLom9kFZrbVzKrNbH47y3ua2dPh8lfMrCBsP8/Mqszs9fD/P3dt+SIi0hkdhr6Z5QAPAhcCI4DpZjaiVbdZwPvufiawELgnbN8FXOzuI4GZwONdVbiIiHReKp/0y4Bqd3/L3Q8AlUBFqz4VwJLw8jJgkpmZu29w9+1h+2agl5n17IrCRUSk81IJ/cHAO0nXa8K2dvu4+0FgD9C/VZ/LgfXu/mHrOzCz2WaWMLNEXV1dqrWLiEgnpWVHrpmdTTDkc317y939YXcvdffSgQMHpqMkEZFYSiX0a4GhSdeHhG3t9jGzXKAvsDu8PgRYDlzl7n8+3oJFROTYpRL664CzzGyYmfUArgRWtuqzkmBHLcAUYLW7u5n1A54F5rv7f3ZV0SIicmw6DP1wjH4usAp4E1jq7pvN7A4zuyTs9gjQ38yqgW8AzYd1zgXOBG43s43h3yld/ihERCQl5u6ZruEwpaWlnkgkMl2GiMgJxcyq3L20o376Ra6ISIwo9EVEYkShLyISIwp9EZEYyc10ASIicbRiQy33rdrK9voGBvXrxbzy4Uwe3Xqyg66n0BcRSbMVG2q59ZnXaWhsAqC2voFbn3kdoNuDX8M7IiJpdt+qrS2B36yhsYn7Vm3t9vtW6IuIpNn2+oZOtXclhb6ISJoN6terU+1dSaEvIpJm88qH0ysv57C2Xnk5zCsf3u33rR25IiJp1ryzVkfviIjExOTRg9MS8q1peEdEJEYU+iIiMaLQFxGJEYW+iEiMKPRFRGJEoS8iEiM6ZFNEul2mZpSUthT6ItKtMjmjpLSl4R0R6VaZnFFS2lLoi0i3yuSMktKWQl9EulUmZ5SUthT6IllkxYZavnD3aobNf5Yv3L2aFRtqM11SRmeUlLa0I1ckS0R1h2kmZ5SUthT6IlniaDtMMx2wmZpRUtpS6KeRjlWW7qQdppKKrAv9qAZrVL96Q7S3WRTriqpB/XpR207Aa4epJMuqHbnNwVpb34BzKFijsDMrqscqR3WbRbWuKNMOU0lFVoV+VIMVovvVO6rbLKp1NYviUTKTRw/mrstGMrhfLwwY3K8Xd102Ut+O5DBZNbwT1WCF6H71juo2i2pdEO2hOu0wlY5k1Sf9KP8IJKpfvaO6zaJaF0T/W4jI0WRV6Ec1WCG6X72jus2iWhdE+1uISEeyangn6j8CieJX76hus6jWBdEdqhNJhbl7pms4TGlpqScSiUyXIXJErcf0IfgWEoVvbhJfZlbl7qUd9cuqT/oi6RDlbyEiHVHoixyDKA7ViaQiq3bkiojI0Sn0RURiJKXQN7MLzGyrmVWb2fx2lvc0s6fD5a+YWUHY3t/MnjezvWb2s64tXUREOqvD0DezHOBB4EJgBDDdzEa06jYLeN/dzwQWAveE7fuB7wDf7LKKRUTkmKXySb8MqHb3t9z9AFAJVLTqUwEsCS8vAyaZmbn7P9z9jwThLyIiGZZK6A8G3km6XhO2tdvH3Q8Ce4D+qRZhZrPNLGFmibq6ulRvJiIinRSJHbnu/rC7l7p76cCBAzNdjohI1kol9GuBoUnXh4Rt7fYxs1ygL7C7KwoUEZGuk0rorwPOMrNhZtYDuBJY2arPSmBmeHkKsNqjNr+DiEhrm5bCwkJY0C/4v2lppivqdh3+ItfdD5rZXGAVkAM86u6bzewOIOHuK4FHgMfNrBp4j+CNAQAz2wZ8AuhhZpOB8919S9c/FBGRTti0FH79P6ExnDxvzzvBdYCiaZmrq5tpwjURiaeFhUHQt9Z3KNz0RvrrOU6pTrgWiR25IiJpt6emc+1dLUNDSwp9EYmnvkM6196VmoeW9rwD+KGhpTQEv0JfROJp0u2Q1+rEN3m9gvbu9twdh/YlNGtsCNq7mUJfROKpaBpc/JNgDB8L/l/8k/TsxM3g0JLm0xeR+CqalpkjdfoOOcJO5O4fWtInfRGRdMvg0JJCX0Qk3TI4tJR9wzublgY7Q/bUBF+VJt2e1T+0yGpRfi6jXJucGDI0tJRdoR/1X9hFNSiiWFeUn8so1ybSgewa3sngYVAdyuBxuSdkXVF+LqNcWwznkpHOya7Qz/Qv7I4mqkER1bqi/FxGtbaovoFLpGRX6GfyF3YdiWpQRLWuKD+XUa0tqm/gEinZFfqZ/IVdR6IaFFGtK8rPZVRri+obuERKdoV+Jn9h15GoBkVU64rycxnV2qL6Bi6RoqmV0ymKR8lEuS7pnNZHFUHwBh6FNyS9xrpdqlMrK/RFskkUwzXKb0ZZJNXQz67j9EXiLlNzyRzN0XYwR63WGMiuMX0RiR7tYI4Uhb6IdC/tYI4Uhb6IdK+oHiEWUwp9EeleUT3ENaa0I1dEul8UdzDHlD7pi4jEiEJfRCRGFPoiIjGi0BcRiRGFvohIjCj0RURiRKEvIhIjCn0RkRhR6IuIxIhCX0QkRhT6IiIxotAXEYkRhb6ISIwo9EVEYkShLyISIwp9EZEYSSn0zewCM9tqZtVmNr+d5T3N7Olw+StmVpC07NawfauZlXdd6SIi0lkdhr6Z5QAPAhcCI4DpZjaiVbdZwPvufiawELgnvO0I4ErgbOAC4KFwfSIikgGpfNIvA6rd/S13PwBUAhWt+lQAS8LLy4BJZmZhe6W7f+jubwPV4fpERCQDUgn9wcA7SddrwrZ2+7j7QWAP0D/F24qISJpEYkeumc02s4SZJerq6jJdjohI1kol9GuBoUnXh4Rt7fYxs1ygL7A7xdvi7g+7e6m7lw4cODD16kVEpFNSCf11wFlmNszMehDsmF3Zqs9KYGZ4eQqw2t09bL8yPLpnGHAW8GrXlC4iIp2V21EHdz9oZnOBVUAO8Ki7bzazO4CEu68EHgEeN7Nq4D2CNwbCfkuBLcBB4F/dvambHouIiHTAgg/k0VFaWuqJRCLTZYiInFDMrMrdSzvqF4kduSIikh4KfRGRGFHoi4jEiEJfRCRGFPoiIjGi0BcRiRGFvohIjCj0RURiRKEvIhIjCn0RkRhR6IuIxIhCX0QkRhT6IiIxErlZNs2sDvhLputIkwHArkwXEQHaDodoWwS0HQ5JdVuc4e4dnoUqcqEfJ2aWSGUq1Gyn7XCItkVA2+GQrt4WGt4REYkRhb6ISIwo9DPr4UwXEBHaDodoWwS0HQ7p0m2hMX0RkRjRJ30RkRhR6IuIxIhCP83MrNjMXjazjWaWMLOysN3M7CdmVm1mm8zsnEzXmg5mdqOZ/V8z22xm9ya13xpui61mVp7JGtPFzG42MzezAeH12L0mzOy+8PWwycyWm1m/pGWxek2Y2QXhY602s/ldtmJ3118a/4DfAReGl/8FWJN0+beAAecCr2S61jRsi4nAH4Ce4fVTwv8jgNeAnsAw4M9ATqbr7eZtMRRYRfDDxAExfk2cD+SGl+8B7onjawLICR/jp4Ee4WMf0RXr1if99HPgE+HlvsD28HIF8JgHXgb6mdlpmSgwjW4A7nb3DwHcfWfYXgFUuvuH7v42UA2UZajGdFkIfIvg9dEsdq8Jd/+dux8Mr74MDAkvx+01UQZUu/tb7n4AqCTYBsdNoZ9+XwfuM7N3gB8Ct4btg4F3kvrVhG3Z7DPAODN7xcxeMLMxYXustoWZVQC17v5aq0Wx2g7tuJbgmw7Eb1t02+PN7YqVyOHM7A/Aqe0sug2YBNzk7v9hZtOAR4AvpbO+dOpgW+QCJxMMXYwBlprZp9NYXtp0sB2+TTCsEQtH2xbu/quwz23AQeDJdNYWBwr9buDuRwxxM3sM+F/h1X8HfhFeriUY1202JGw7oXWwLW4AnvFgEPNVM/uIYHKprNsWR9oOZjaSYIz6NTOD4LGuD3fwZ912gKO/JgDM7GrgImBS+NqALN0WR9Ftj1fDO+m3HRgfXv5n4L/CyyuBq8IjNs4F9rj7u5koMI1WEOzMxcw+Q7DDahfBtrjSzHqa2TDgLODVjFXZjdz9dXc/xd0L3L2A4Gv8Oe7+N2L4mjCzCwj2bVzi7vuSFsXmNRFaB5xlZsPMrAdwJcE2OG76pJ9+XwUeMLNcYD8wO2z/DcHRGtXAPuCazJSXVo8Cj5rZG8ABYGb4yW6zmS0FthB8xf9Xd2/KYJ2ZEsfXxM8IjtD5ffjN52V3n+PusXpNuPtBM5tLcERXDvCou2/uinVrGgYRkRjR8I6ISIwo9EVEYkShLyISIwp9EZEYUeiLiMSIQl9EJEYU+iIiMfL/AQRoBU2A9O3aAAAAAElFTkSuQmCC\n",
      "text/plain": [
       "<Figure size 432x288 with 1 Axes>"
      ]
     },
     "metadata": {
      "needs_background": "light"
     },
     "output_type": "display_data"
    }
   ],
   "source": [
    "plt.scatter(np.log(thetas), tv)\n",
    "plt.scatter(np.log(thetas), tp)\n",
    "plt.xlabel = \"Log of theta\"\n",
    "plt.ylabel = \"Time\"\n",
    "plt.title(\"Runtime vs theta\")\n",
    "plt.legend([\"Value iteration\", \"Policy iteration\"])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Observations\n",
    "\n",
    "- **Value iteration**: Takes more time but less number of episodes to converge\n",
    "- **Policy iteration**: Takes less time but more number of episodes to converge"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.7.1"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
